import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;

import java.io.*;
import java.lang.reflect.Array;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;

public class CachingYaSearchEngine extends YaSearchEngine {
    public CachingYaSearchEngine(String queryCache, String docCache, String htmlDirectory) throws IOException, UnsupportedEncodingException {
        this.linkToFile = new HashMap<String,String>();
        this.htmlDirectory = new File(htmlDirectory);
        FileInputStream docCacheStream = new FileInputStream(docCache);
        BufferedReader docCacheReader = new BufferedReader(new InputStreamReader(docCacheStream, "UTF-8"));
        String uri;
        while ((uri = docCacheReader.readLine()) != null)   {
            String file = docCacheReader.readLine();
            linkToFile.put(uri,file);
        }
        docCacheReader.close();
        docCacheStream.close();
        this.linkToFileStream = new FileOutputStream(docCache, true);
        this.linkToFileWriter = new OutputStreamWriter(this.linkToFileStream,"UTF-8");

        queryToLinks = new HashMap();

        FileInputStream queryCacheStream = new FileInputStream(queryCache);
        BufferedReader queryCacheReader = new BufferedReader(new InputStreamReader(queryCacheStream, "UTF-8"));
        String query;
        while ((query = queryCacheReader.readLine()) != null)   {
            ArrayList<String> links = new ArrayList<String>();
            int n = Integer.parseInt(queryCacheReader.readLine());
            while (n > 0){
                links.add(queryCacheReader.readLine());
                n--;
            }
            queryToLinks.put(query,links);
        }
        queryCacheReader.close();
        queryCacheStream.close();

        this.queryCacheStream = new FileOutputStream(queryCache, true);
        this.queryCacheWriter = new OutputStreamWriter(this.queryCacheStream,"UTF-8");
    }

    File htmlDirectory;

    FileOutputStream queryCacheStream;
    OutputStreamWriter queryCacheWriter;
    FileOutputStream linkToFileStream;
    OutputStreamWriter linkToFileWriter;


    HashMap<String,List<String>> queryToLinks;
    HashMap<String,String> linkToFile;
    boolean isCacheOnly = false;

    public void setCacheOnly(boolean value){
        isCacheOnly = value;
    }
    public boolean getCacheOnly(){
        return isCacheOnly;
    }

    @Override
    protected List<String> getLinks(String query) throws IOException {
        if (isCacheOnly && !queryToLinks.containsKey(query)) return new ArrayList<String>();

        if (!queryToLinks.containsKey(query)) {
            List<String> links = super.getLinks(query);
            queryToLinks.put(query, links);
            queryCacheWriter.write(query + "\n");
            queryCacheWriter.write(new Integer(links.size()).toString() + "\n");
            for (String link : links){
                queryCacheWriter.write(link + "\n");
            }
            queryCacheWriter.flush();
            queryCacheStream.flush();
        }

        return queryToLinks.get(query);
    }

    @Override
    protected void getDocument(String uri, ArrayList<WebDocument> docs) throws Exception {
        if (uri.endsWith(".pdf")) return;
        if (uri.endsWith(".xls")) return;
        //System.out.println(uri);
        // jsoup bug
        if (uri.contains("http://bashne.net/")) return;
        if (uri.contains("http://www.oblvoin.ru/vopros2009.html")) return;

        if (isCacheOnly && !linkToFile.containsKey(uri)) return;

        if (!linkToFile.containsKey(uri)) {
            File file = new File(htmlDirectory, new Integer(linkToFile.size()).toString() + ".html");
            try{
                Document html = Jsoup.parse(new URL(uri), 3000);
                FileOutputStream htmlStream = new FileOutputStream(file.getPath(), false);
                OutputStreamWriter htmlWriter = new OutputStreamWriter(htmlStream, "UTF-8");
                htmlWriter.write(html.html());
                htmlWriter.flush();
                htmlStream.flush();
                htmlWriter.close();
                htmlStream.close();
                linkToFileWriter.write(uri + "\n");
                linkToFileWriter.write(file.getPath() + "\n");
                linkToFileWriter.flush();
                linkToFileStream.flush();
                linkToFile.put(uri,file.getPath());
            }
            catch(Exception e)
            {
                badQueries++;
                if (badQueries > 700) throw new Exception();
            }
        }

        if (linkToFile.containsKey(uri)) {
            try{
                FileInputStream stream = new FileInputStream(linkToFile.get(uri));
                BufferedReader reader = new BufferedReader(new InputStreamReader(stream, "UTF-8"));
                String line;
                StringBuffer content = new StringBuffer();
                while ((line = reader.readLine())!=null) {
                    content.append(line + "\n");
                }
                stream.close();
                WebDocument doc = WebDocument.createFromHtml(content.toString());
                docs.add(doc);
                goodQueries++;
            }catch(Exception e){
                badQueries++;
            }
        }
    }

    public void dispose() throws IOException {
        queryCacheWriter.close();
        queryCacheStream.close();
        linkToFileStream.close();
        linkToFileWriter.close();
    }
}
